package main.kotlin.webMagic


import us.codecraft.webmagic.selector.PlainText
import us.codecraft.webmagic.ResultItems
import us.codecraft.webmagic.Task
import us.codecraft.webmagic.pipeline.Pipeline
import main.kotlin.singlePage.ImageDownloader
import main.kotlin.markdown.*
import org.apache.commons.io.FilenameUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Node
import org.jsoup.nodes.TextNode
import java.io.File
import kotlin.text.Regex
import mu.KotlinLogging
import org.jsoup.nodes.Document
import java.util.Scanner

private  val logger = KotlinLogging.logger{}

class DGEPagePipeline: Pipeline {
    private val downloader = ImageDownloader()
    private val log = DownloadLog()


    override fun process(resultItems: ResultItems, p1: Task) {
        val content = resultItems.get<String>("content") ?: ""

        val soup = Jsoup.parse(content)
        val date = resultItems.get<String>("date")
        var title = resultItems.get<PlainText>("title").toString()

        val pattern = Regex(" - .* \\| .* - \\w+\\.\\w+\$")
        title = pattern.replace(title, "")

        println(title)

        title = Regex("[/:*?\"<>|]").replace(title, "")

        val outDir = File(date, title)
        val imgDir = File(outDir, "img")

        // check output directory
        if ( !imgDir.exists() ) { imgDir.mkdirs() }

        // init results class
        val wholePage = PageText(title)
        wholePage.add(this.getAllElements(
                soup.select("*").first().childNodes(),
                outDir.toString(),
                resultItems.request.url)
        )

        wholePage.writeToMd(File(outDir, "$title.md").toString())

        // save html
        MdToHTML(wholePage, "self").writeTo(File(outDir, "$title.html").toString())

        log.connect()
        log.insert(resultItems.request.url, title)
        log.close()
    }

    /**
     * 通过递归，收集某个标签之内的所有文本、链接和图片
     * @param tags: 一列表的Node
     */
    private fun getAllElements(tags: List<Node>, outdir: String, title: String = "1024", url:String = ""): PageText {
        val wholePage = PageText(title)
        wholePage.url = url

        for (t in tags) {
            if (t.childNodeSize() > 0) { // 递归遍历子节点
                wholePage.add(this.getAllElements(t.childNodes(), outdir))
            } else if (t.nodeName() == "a") {
                // 获取链接
                wholePage.addLink(
                        this.getTextofLink(t),
                        this.getAbsolutePath(t, "href")
                )
            } else if (t.nodeName() in listOf("img", "input")) {
                // 获取图片
                if (t.attr("src") != null && t.attr("src") != "") {
                    val imgLink = this.getAbsolutePath(t, "src")

                    val imgFile = File(File(outdir, "img"), FilenameUtils.getName(imgLink)).toString()

                    wholePage.addImg(
                            this.getTextofLink(t), imgFile.replace(outdir, ".")

                    )

                    // 下载图片
                    try{
                        this.downloader.download(imgLink, imgFile, false)
                    } catch (e: Exception) {
                        logger.error { "URL: $url" }
                        logger.error { "Error ${e.message}" }
                    }

                }
            } else if (t is TextNode){
                // 获取标题
                if (t.nodeName() in listOf("h1", "h2", "h3", "h4", "h5", "h6")) {
                    wholePage.addHeading(
                            t.text().trim(), getHeadLevel(t.outerHtml())
                    )
                } else {  // 获取其他普通文本
                    val text = t.text().trim()

                    if (text != "") {
                        wholePage.addText(text)
                    }
                }


            }
        }
        return wholePage
    }

    /**
     * 通过正则提取heading标签的等级
     * @param html 字符串
     */
    private fun getHeadLevel(html: String): Int {
        var level = 0
        val pattern = Regex("<h\\d[^>]*>")

        val value: String? = pattern.find(html)?.groups?.get(0)?.value

        if (value != null) {
            level = value.replace(">", "").replace("<h", "").toInt()
        }

        return level
    }


    /**
     * @param tag 输入的标签，获取绝对路径的
     * @param attr 属性的名称，通常不是href就是src
     */
    private fun getAbsolutePath(tag: Node, attr: String): String {
        val value: String = tag.absUrl(attr)

        when (value != "") {
            true -> return value
            false -> return tag.attr(attr)
        }
    }


    /**
     * 分为两类TextNode，和DataNode，分别处理， TextNode直接获取text，DataNode则尝试获取alt属性
     * @param tag: Node
     */
    private fun getTextofLink(tag: Node): String {
        when (tag.nodeName() == "a") {
            true -> {
                var text = (tag as TextNode).text()

                if ( text == "" ) {
                    text = FilenameUtils.getName(tag.absUrl("href"))
                }
                return text
            }
            false -> {
                return when (tag.hasAttr("alt")) {
                    true -> tag.attr("alt")
                    false -> FilenameUtils.getName(tag.absUrl("src"))
                }
            }
        }
    }
}
